p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] test.py (python)
from transformers import AutoTokenizer

tok = AutoTokenizer.from_pretrained(
    "Qwen/Qwen3-0.6B",
    trust_remote_code=True,
)

messages = [
    {"role": "system", "content": "You are helpful."},
    {"role": "user", "content": "What is 2 + 2?"},
    {"role": "assistant", "content": "<think>Hm. I think 2 + 2 must be 4</think>The answer is 4."},
]

print("=== NO THINKING ===")
print(tok.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=False,
    enable_thinking=False # Switches between thinking and non-thinking modes. Default is True.
))

print("
=== DEFAULT (thinking) ===")
print(tok.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=False,
))
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/evaluate_qwen0.6_handcrafted.sh (shellscript)
# this is a demo script how to evaluate qwen0.6b on our handcrafted dataset

# Ensure that you sbatched `slurm/dev/mihir/berlin/crowd_pilot/evaluate/launch_judge_sglang.sh` and the judge is running on hai004
# This script assumes that you are in a salloc session at hai007
generator_host="hai007"
generator_port=30000
generator_model_path="Qwen/Qwen3-0.6B"

judge_host="hai004"
judge_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"

set -x

source .venv/bin/activate
module load CUDA/12.8

# preprocess the testcases from md to jsonl. 
# In this step the script converts the eval into multistep convo format, with the <EVAL> turn as the last one.
# The prompt template etc is chosen here.
python src/input_pipeline/trace_to_testcases.py

# Here we launch an sglang server with the model-to-evaluate. 
# We let the model generate responses to the prompts in our eval and compare it to the gt answer using string similarity.
context_length=40960
python src/evaluation/sglang_generations.py \
  --model_path $generator_model_path \
  --context_length $context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --concurrency 32 \


# Since string similarity is crude, we prompt an LLM-judge to compare the generated answer to the gt answer semantically.
# By default we launch an sglang server with Qwen-Coder-30B-A3B-Instruct. This model is not a thinker!
accept_threshold=0.7
python src/evaluation/sglang_eval.py \
  --model_path $judge_model_path \
  --accept_threshold $accept_threshold \
  --server_host $judge_host \
  --prompt_file data/prompts/command_evaluation_prompt.txt \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/evaluate_qwen0.6_unit_test.sh (shellscript)
# this is a demo script how to evaluate qwen0.6b on our handcrafted dataset

# Ensure that you sbatched `slurm/dev/mihir/berlin/crowd_pilot/evaluate/launch_judge_sglang.sh` and the judge is running on hai004
# This script assumes that you are in a salloc session at hai007
generator_host="hai007"
generator_port=30000
generator_model_path="Qwen/Qwen3-0.6B"

judge_host="hai004"
judge_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"

set -x

source .venv/bin/activate
module load CUDA/12.8

# preprocess the testcases from md to jsonl. 
# In this step the script converts the eval into multistep convo format, with the <EVAL> turn as the last one.
# The prompt template etc is chosen here.
python src/input_pipeline/trace_to_testcases.py \
  --input_dir data/eval/unit_tests \
  --output_file data/eval/handcrafted_test_cases/unit_test_cases.jsonl

# Here we launch an sglang server with the model-to-evaluate. 
# We let the model generate responses to the prompts in our eval and compare it to the gt answer using string similarity.
context_length=40960
python src/evaluation/sglang_generations.py \
  --input_file data/eval/handcrafted_test_cases/unit_test_cases.jsonl \
  --output_file data/eval/generations/unit_test_cases_generations.json \
  --system_prompt_file data/prompts/crowd_pilot_prompt.txt \
  --model_path $generator_model_path \
  --context_length $context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --concurrency 32 \


# Since string similarity is crude, we prompt an LLM-judge to compare the generated answer to the gt answer semantically.
# By default we launch an sglang server with Qwen-Coder-30B-A3B-Instruct. This model is not a thinker!
accept_threshold=0.7
python src/evaluation/sglang_eval.py \
  --input_file data/eval/generations/unit_test_cases_generations.json \
  --output_file data/eval/generations/unit_test_cases_evaluation.json \
  --model_path $judge_model_path \
  --accept_threshold $accept_threshold \
  --server_host $judge_host \
  --prompt_file data/prompts/command_evaluation_prompt.txt \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/base_models/qwen_1.7B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_qwen_1.7B


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
generations_file="data/eval/output/base_models/qwen_1.7B_generations.jsonl"
evaluations_file="data/eval/output/base_models/qwen_1.7B_evaluations.jsonl"
mkdir -p $(dirname $generations_file)

# Evaluation
judge_host="jwb0033i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B"
wandb_tags="baseline qwen_1.7B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/base_models/qwen_4B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_qwen_4B


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
generations_file="data/eval/output/base_models/qwen_4B_generations.jsonl"
evaluations_file="data/eval/output/base_models/qwen_4B_evaluations.jsonl"
mkdir -p $(dirname $generations_file)

# Evaluation
judge_host="jwb0033i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_4B"
wandb_tags="baseline qwen_4B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \


python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/evaluate/evaluate_qwen_0.6B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=00:30:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=booster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=gen_eval_qwen_0.6B


source .venv/bin/activate

module load Stages/2026 CUDA/13

export WANDB_MODE=offline

cat "$0"



# Generation
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--qwen--qwen2.5-0.5b-instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775"
context_length=20480


# Evaluation
judge_port=30000
judge_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--qwen--qwen2.5-0.5b-instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775"
judge_context_length=20480
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="${SLURM_JOB_NAME}_${SLURM_JOB_ID}"
wandb_tags="qwen2.5_0.6b debug"
wandb_eval_type="handcrafted"

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
generations_file="data/eval/handcrafted_test_cases/handcrafted_0.6B_debug_generations.jsonl"
evaluations_file="data/eval/handcrafted_test_cases/handcrafted_0.6B_debug_evaluations.jsonl"
# generations_file="data/eval/handcrafted_test_cases/handcrafted_${wandb_name}_generations.jsonl"
# evaluations_file="data/eval/handcrafted_test_cases/handcrafted_${wandb_name}_evaluations.jsonl"



python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_port $generator_port \
  --concurrency 32 \
  --tp-size 1


python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --tp-size 1
[tab] slurm/dev/alfred/juwels/evals/lora_runs/eval_qwen_spawner.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_lora_runs_qwen_1.7B_ckpt_13032800


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"

model_size=$MODEL_SIZE
run_id=$RUN_ID

# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path=$GENERATOR_MODEL_PATH
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/${model_size}/${run_id}/"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_${model_size}_${run_id}_generations.jsonl"
evaluations_file="${output_dir}/qwen_${model_size}_${run_id}_evaluations.jsonl"

mkdir -p $output_dir

echo "Running evaluation for $model_size $run_id"
echo "Generator model path: $generator_model_path"
echo "Lora path: $lora_path"
echo "Generations file: $generations_file"
echo "Evaluations file: $evaluations_file"



# Evaluation
judge_host="jwb0033i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="eval_qwen_${model_size}_${run_id}"
wandb_tags="lora_run qwen_${model_size}"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/lora_runs/copy_adapters.sh (shellscript)
#!/bin/bash

# Function to get the latest checkpoint iteration
get_latest_iter() {
    local ckpt_dir="$1/checkpoints"
    ls -1 "$ckpt_dir" 2>/dev/null | grep -E '^iter_[0-9]+$' | sort -t_ -k2 -n | tail -1
}

# list of ckpts 1.7B
ckpts_1_7B=(
13032800
13032802
)


for ckpt_id in ${ckpts_1_7B[@]}; do
    base_dir="/p/scratch/envcomp/mihir/huggingface/shared_data/${ckpt_id}"
    latest_iter=$(get_latest_iter "$base_dir")
    src="${base_dir}/checkpoints/${latest_iter}"
    dst="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/1.7B/${ckpt_id}"
    echo "Copying ${ckpt_id} (${latest_iter}) -> ${dst}"
    mkdir -p $dst
    cp $src/adapter_* $dst/
done



ckpts_4B=(
13032809
13032811
13032810
)

for ckpt_id in ${ckpts_4B[@]}; do
    base_dir="/p/scratch/envcomp/mihir/huggingface/shared_data/${ckpt_id}"
    latest_iter=$(get_latest_iter "$base_dir")
    src="${base_dir}/checkpoints/${latest_iter}"
    dst="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/4B/${ckpt_id}"
    echo "Copying ${ckpt_id} (${latest_iter}) -> ${dst}"
    mkdir -p $dst
    cp $src/adapter_* $dst/
done




ckpts_8B=(
13032817
13032815
13032816
)
for ckpt_id in ${ckpts_8B[@]}; do
    base_dir="/p/scratch/envcomp/mihir/huggingface/shared_data/${ckpt_id}"
    latest_iter=$(get_latest_iter "$base_dir")
    src="${base_dir}/checkpoints/${latest_iter}"
    dst="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/8B/${ckpt_id}"
    echo "Copying ${ckpt_id} (${latest_iter}) -> ${dst}"
    mkdir -p $dst
    cp $src/adapter_* $dst/
done



ckpts_32B=(
13032807
)

for ckpt_id in ${ckpts_32B[@]}; do
    base_dir="/p/scratch/envcomp/mihir/huggingface/shared_data/${ckpt_id}"
    latest_iter=$(get_latest_iter "$base_dir")
    src="${base_dir}/checkpoints/${latest_iter}"
    dst="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/32B/${ckpt_id}"
    echo "Copying ${ckpt_id} (${latest_iter}) -> ${dst}"
    mkdir -p $dst
    cp $src/adapter_* $dst/
done
[tab] slurm/dev/alfred/juwels/evals/lora_runs/eval_qwen_spawner.sh (shellscript)
#!/usr/bin/env bash

load_ckpts=(
    # "1.7B/13032800"
    # "1.7B/13032802"
    "4B/13032809"
    "4B/13032811"
    "4B/13032810"
    "8B/13032817"
    "8B/13032815"
    "8B/13032816"
    "32B/13032807"
)

qwen_1_7B_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
qwen_4B_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-4B-Instruct-2507/snapshots/cdbee75f17c01a7cc42f958dc650907174af0554"
qwen_8B_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-8B/snapshots/b968826d9c46dd6066d109eabc6255188de91218"
qwen_32B_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-32B/snapshots/9216db5781bf21249d130ec9da846c4624c16137"

for ckpt in "${load_ckpts[@]}"; do
    model_size=$(echo $ckpt | cut -d'/' -f1)
    run_id=$(echo $ckpt | cut -d'/' -f2)
    export MODEL_SIZE=$(echo $ckpt | cut -d'/' -f1)
    export RUN_ID=$(echo $ckpt | cut -d'/' -f2)
    
    if [ "$MODEL_SIZE" == "1.7B" ]; then
        export GENERATOR_MODEL_PATH=$qwen_1_7B_model_path
    elif [ "$MODEL_SIZE" == "4B" ]; then
        export GENERATOR_MODEL_PATH=$qwen_4B_model_path
    elif [ "$MODEL_SIZE" == "8B" ]; then
        export GENERATOR_MODEL_PATH=$qwen_8B_model_path
    elif [ "$MODEL_SIZE" == "32B" ]; then
        export GENERATOR_MODEL_PATH=$qwen_32B_model_path
    elif [ "$MODEL_SIZE" == "32B" ]; then
        export GENERATOR_MODEL_PATH=$qwen_32B_model_path
    fi

    sbatch --export=ALL --job-name=eval_qwen_${model_size}_${run_id} \
        slurm/dev/alfred/juwels/evals/lora_runs/eval_qwen_spawner.sbatch

done
[tab] slurm/jobs/mihir/berlin/crowd-pilot/dev_fleet/run_sqwen3-0.6b-sft-torch-crowd-code.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-1.7Bb-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=run_qwen3-1.6b-sft-torch-crowd-code


trap 'echo "Job killed by signal"' USR1

cat $0
module load CUDA/12.8

source .venv/bin/activate

srun sh slurm/jobs/mihir/berlin/crowd-pilot/dev_fleet/qwen_1.7b_lora_bs_32.sh
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_lora_bs_16.sh (shellscript)
#!/bin/bash
#
# Ray-free SFT Training Script
#

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
# FIXME(f.srambical): this is hardcoded for now
GPUS_PER_NODE=4
NUM_NODES=${SLURM_JOB_NUM_NODES}
NODE_RANK=${SLURM_NODEID}
MASTER_ADDR=${MASTER_ADDR:-$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)}

NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l)
if [ "$NVLINK_COUNT" -gt 0 ]; then
    HAS_NVLINK=1
else
    HAS_NVLINK=0
fi
echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

RUN_ID=${RUN_ID:-$SLURM_JOBID}
RUN_NAME="qwen1.7B_lora_bs_16_${RUN_ID}"
WAVE_NAME="ANTA_CLAUS"
LOAD_PATH="/p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B"
SAVE_PATH="/p/scratch/envcomp/mihir/huggingface/shared_data/${RUN_ID}/checkpoints"

CKPT_ARGS=(
   --hf-checkpoint /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
   --load ${LOAD_PATH}
   --ref-load /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
   --save ${SAVE_PATH}
   --save-interval 1000
)

SFT_ARGS=(
   --rollout-function-path miles.rollout.sft_rollout.generate_rollout
   --prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens.parquet
   --val-prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens_validation.parquet
   --val-interval 1000
   --val-steps 100
   --input-key messages
   --apply-chat-template
   --rollout-shuffle
   --num-rollout 10000
   --rollout-batch-size 16
   --global-batch-size 16

   --loss-type sft_loss
   --calculate-per-token-loss
   --disable-compute-advantages-and-returns
)

LORA_ARGS=(
    --use-lora
    --lora-rank 8
    --lora-alpha 16
    --lora-dropout 0.0
    --lora-target-modules q_proj v_proj
)

OPTIMIZER_ARGS=(
   --optimizer adam
   --lr 1e-4
   --lr-decay-style WSD
   --lr-wsd-decay-style linear
   --lr-warmup-iters 500
   --lr-decay-iters 10000
   --lr-wsd-decay-iters 2000
   --weight-decay 0.1
   --adam-beta1 0.9
   --adam-beta2 0.98
)

WANDB_ARGS=(
   --use-wandb
   --wandb-project crowd-pilot-miles
   --wandb-team instant-uv
   --wandb-group $RUN_NAME
   --wandb-run-id $RUN_ID
   --wandb-dir "/p/scratch/envcomp/logs/${WAVE_NAME}"
)

TRAIN_BACKEND_ARGS=(
   --train-backend fsdp
   --update-weight-buffer-size 536870912
   --attn-implementation flash_attention_3
)

PERF_ARGS=(
   --micro-batch-size 1
)

MISC_ARGS=(
   --rollout-max-context-len 4100
   --rollout-max-prompt-len 4096
   --rollout-max-response-len 4100
   --dump-details /p/scratch/envcomp/mihir/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details
)
echo ${WANDB_ARGS[@]}

torchrun \
    --nproc_per_node=${GPUS_PER_NODE} \
    --nnodes=${NUM_NODES} \
    --node_rank=${NODE_RANK} \
    --master_addr=${MASTER_ADDR} \
    --master_port=${MASTER_PORT:-29500} \
    train_sft.py \
    ${CKPT_ARGS[@]} \
    ${SFT_ARGS[@]} \
    ${LORA_ARGS[@]} \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${TRAIN_BACKEND_ARGS[@]} \
    ${PERF_ARGS[@]} \
    ${MISC_ARGS[@]}
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_no_lora_bs_8.sh (shellscript)
#!/bin/bash
#
# Ray-free SFT Training Script
#

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
# FIXME(f.srambical): this is hardcoded for now
GPUS_PER_NODE=4
NUM_NODES=${SLURM_JOB_NUM_NODES}
NODE_RANK=${SLURM_NODEID}
MASTER_ADDR=${MASTER_ADDR:-$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)}

NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l)
if [ "$NVLINK_COUNT" -gt 0 ]; then
    HAS_NVLINK=1
else
    HAS_NVLINK=0
fi
echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

RUN_ID=${RUN_ID:-$SLURM_JOBID}
RUN_NAME="qwen1.7B_no_lora_bs_8_${RUN_ID}"
WAVE_NAME="ANTA_CLAUS"
LOAD_PATH="/p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B"
SAVE_PATH="/p/scratch/envcomp/mihir/huggingface/shared_data/${RUN_ID}/checkpoints"

CKPT_ARGS=(
   --hf-checkpoint /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
   --load ${LOAD_PATH}
   --ref-load /p/scratch/envcomp/mihir/huggingface/Qwen3-1.7B
   --save ${SAVE_PATH}
   --save-interval 1000
)

SFT_ARGS=(
   --rollout-function-path miles.rollout.sft_rollout.generate_rollout
   --prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens.parquet
   --val-prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens_validation.parquet
   --val-interval 2000
   --val-steps 100
   --input-key messages
   --apply-chat-template
   --rollout-shuffle
   --num-rollout 20000
   --rollout-batch-size 8
   --global-batch-size 8

   --loss-type sft_loss
   --calculate-per-token-loss
   --disable-compute-advantages-and-returns
)

LORA_ARGS=(
    # --use-lora
    # --lora-rank 8
    # --lora-alpha 16
    # --lora-dropout 0.0
    # --lora-target-modules q_proj v_proj
)

OPTIMIZER_ARGS=(
   --optimizer adam
   --lr 1e-5
   --lr-decay-style WSD
   --lr-wsd-decay-style linear
   --lr-warmup-iters 1000
   --lr-decay-iters 20000
   --lr-wsd-decay-iters 4000
   --weight-decay 0.1
   --adam-beta1 0.9
   --adam-beta2 0.98
)

WANDB_ARGS=(
   --use-wandb
   --wandb-project crowd-pilot-miles
   --wandb-team instant-uv
   --wandb-group $RUN_NAME
   --wandb-run-id $RUN_ID
   --wandb-dir "/p/scratch/envcomp/logs/${WAVE_NAME}"
)

TRAIN_BACKEND_ARGS=(
   --train-backend fsdp
   --update-weight-buffer-size 536870912
   --attn-implementation flash_attention_3
)

PERF_ARGS=(
   --micro-batch-size 1
)

MISC_ARGS=(
   --rollout-max-context-len 4100
   --rollout-max-prompt-len 4096
   --rollout-max-response-len 4100
   --dump-details /p/scratch/envcomp/mihir/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details
)
echo ${WANDB_ARGS[@]}

torchrun \
    --nproc_per_node=${GPUS_PER_NODE} \
    --nnodes=${NUM_NODES} \
    --node_rank=${NODE_RANK} \
    --master_addr=${MASTER_ADDR} \
    --master_port=${MASTER_PORT:-29500} \
    train_sft.py \
    ${CKPT_ARGS[@]} \
    ${SFT_ARGS[@]} \
    ${LORA_ARGS[@]} \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${TRAIN_BACKEND_ARGS[@]} \
    ${PERF_ARGS[@]} \
    ${MISC_ARGS[@]}
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_qwen_1.7B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_lora_runs_qwen_1.7B_ckpt_13032800


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/13032800/checkpoints/iter_0010000"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_1.7B_ckpt_13032800_generations.jsonl"
evaluations_file="${output_dir}/qwen_1.7B_ckpt_13032800_evaluations.jsonl"

mkdir -p $output_dir

# Evaluation
judge_host="jwb0129i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B_ckpt_13032800"
wandb_tags="lora_run qwen_1.7B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_lora_runs_qwen_1.7B_ckpt_13032800


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/13032800/checkpoints/iter_0010000"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_1.7B_ckpt_13032800_generations.jsonl"
evaluations_file="${output_dir}/qwen_1.7B_ckpt_13032800_evaluations.jsonl"

mkdir -p $output_dir

# Evaluation
judge_host="jwb0129i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B_ckpt_13032800"
wandb_tags="lora_run qwen_1.7B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/eval_miles_run.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_lora_runs_qwen_1.7B_ckpt_13032800


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/13032800/checkpoints/iter_0010000"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_1.7B_ckpt_13032800_generations.jsonl"
evaluations_file="${output_dir}/qwen_1.7B_ckpt_13032800_evaluations.jsonl"

mkdir -p $output_dir

# Evaluation
judge_host="jwb0129i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B_ckpt_13032800"
wandb_tags="lora_run qwen_1.7B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/sglang_eval_server.sh (shellscript)
#! /bin/bash

module load CUDA/12.8
source .venv/bin/activate



model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"

accept_threshold=0.7
server_host="hai004"


python src/evaluation/sglang_eval.py \
  --model_path $model_path \
  --accept_threshold $accept_threshold \
  --server_host $server_host \
  --prompt_file data/prompts/command_evaluation_prompt.txt \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/handcrafted/sglang_generate_server.sh (shellscript)
#! /bin/bash

module load CUDA/12.8
source .venv/bin/activate


# model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"
# model_path="Qwen/Qwen3-0.6B"
# context_length=128000

server_host="hai007"
server_port=30000
model_path="Qwen/Qwen3-0.6B"
context_length=40960


python src/evaluation/sglang_generations.py \
  --model_path $model_path \
  --context_length $context_length \
  --server_host $server_host \
  --server_port $server_port \
  --concurrency 32 \
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/evaluate_qwen8b_unit_test.sh (shellscript)
# this is a demo script how to evaluate qwen0.6b on our handcrafted dataset

# Ensure that you sbatched `slurm/dev/mihir/berlin/crowd_pilot/evaluate/launch_judge_sglang.sh` and the judge is running on hai004
# This script assumes that you are in a salloc session at hai007
generator_host="hai007"
generator_port=30000
generator_model_path="Qwen/Qwen3-0.6B"

judge_host="hai004"
judge_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"

set -x

source .venv/bin/activate
module load CUDA/12.8

# preprocess the testcases from md to jsonl. 
# In this step the script converts the eval into multistep convo format, with the <EVAL> turn as the last one.
# The prompt template etc is chosen here.
python src/input_pipeline/trace_to_testcases.py \
  --input_dir data/eval/unit_tests \
  --output_file data/eval/handcrafted_test_cases/unit_test_cases.jsonl

# Here we launch an sglang server with the model-to-evaluate. 
# We let the model generate responses to the prompts in our eval and compare it to the gt answer using string similarity.
context_length=40960
python src/evaluation/sglang_generations.py \
  --input_file data/eval/handcrafted_test_cases/unit_test_cases.jsonl \
  --output_file data/eval/generations/unit_test_cases_generations.json \
  --system_prompt_file data/prompts/crowd_pilot_prompt.txt \
  --model_path $generator_model_path \
  --context_length $context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --concurrency 32 \


# Since string similarity is crude, we prompt an LLM-judge to compare the generated answer to the gt answer semantically.
# By default we launch an sglang server with Qwen-Coder-30B-A3B-Instruct. This model is not a thinker!
accept_threshold=0.7
python src/evaluation/sglang_eval.py \
  --input_file data/eval/generations/unit_test_cases_generations.json \
  --output_file data/eval/generations/unit_test_cases_evaluation.json \
  --model_path $judge_model_path \
  --accept_threshold $accept_threshold \
  --server_host $judge_host \
  --prompt_file data/prompts/command_evaluation_prompt.txt \
  --no_launch_server
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/launch_judge_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --job-name=run_judge
#SBATCH --qos=low

module load CUDA/12.8
FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1 

# model_id="qwen/qwen2.5-0.5b-instruct"
model_id="Qwen/Qwen3-Coder-30B-A3B-Instruct"
# model_id="Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0