p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_lora_bs_32.sh (shellscript)
#!/bin/bash
#
# Ray-free SFT Training Script
#

export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
# FIXME(f.srambical): this is hardcoded for now
GPUS_PER_NODE=4
NUM_NODES=${SLURM_JOB_NUM_NODES}
NODE_RANK=${SLURM_NODEID}
MASTER_ADDR=${MASTER_ADDR:-$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)}

NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l)
if [ "$NVLINK_COUNT" -gt 0 ]; then
    HAS_NVLINK=1
else
    HAS_NVLINK=0
fi
echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

RUN_ID=${RUN_ID:-$SLURM_JOBID}
RUN_NAME="qwen1.7B_lora_bs_32_${RUN_ID}"
WAVE_NAME="ANTA_CLAUS"
LOAD_PATH="/p/scratch/envcomp/mihir/huggingface/Qwen3-4B"
SAVE_PATH="/p/scratch/envcomp/mihir/huggingface/shared_data/${RUN_ID}/checkpoints"

CKPT_ARGS=(
   --hf-checkpoint /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
   --load ${LOAD_PATH}
   --ref-load /p/scratch/envcomp/mihir/huggingface/Qwen3-4B
   --save ${SAVE_PATH}
   --save-interval 500
)

SFT_ARGS=(
   --rollout-function-path miles.rollout.sft_rollout.generate_rollout
   --prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens.parquet
   --val-prompt-data /p/scratch/envcomp/mihir/huggingface/nemo_hf_part_jsonl_4k_tokens_validation.parquet
   --val-interval 500
   --val-steps 100
   --input-key messages
   --apply-chat-template
   --rollout-shuffle
   --num-rollout 5000
   --rollout-batch-size 32
   --global-batch-size 32

   --loss-type sft_loss
   --calculate-per-token-loss
   --disable-compute-advantages-and-returns
)

LORA_ARGS=(
    --use-lora
    --lora-rank 8
    --lora-alpha 16
    --lora-dropout 0.0
    --lora-target-modules q_proj v_proj
)

OPTIMIZER_ARGS=(
   --optimizer adam
   --lr 1e-4
   --lr-decay-style WSD
   --lr-wsd-decay-style linear
   --lr-warmup-iters 250
   --lr-decay-iters 5000
   --lr-wsd-decay-iters 1000
   --weight-decay 0.1
   --adam-beta1 0.9
   --adam-beta2 0.98
)

WANDB_ARGS=(
   --use-wandb
   --wandb-project crowd-pilot-miles
   --wandb-team instant-uv
   --wandb-group $RUN_NAME
   --wandb-run-id $RUN_ID
   --wandb-dir "/p/scratch/envcomp/logs/${WAVE_NAME}"
)

TRAIN_BACKEND_ARGS=(
   --train-backend fsdp
   --update-weight-buffer-size 536870912
   --attn-implementation flash_attention_3
)

PERF_ARGS=(
   --micro-batch-size 1
)

MISC_ARGS=(
   --rollout-max-context-len 4100
   --rollout-max-prompt-len 4096
   --rollout-max-response-len 4100
   --dump-details /p/scratch/envcomp/mihir/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details
)
echo ${WANDB_ARGS[@]}

torchrun \
    --nproc_per_node=${GPUS_PER_NODE} \
    --nnodes=${NUM_NODES} \
    --node_rank=${NODE_RANK} \
    --master_addr=${MASTER_ADDR} \
    --master_port=${MASTER_PORT:-29500} \
    train_sft.py \
    ${CKPT_ARGS[@]} \
    ${SFT_ARGS[@]} \
    ${LORA_ARGS[@]} \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${TRAIN_BACKEND_ARGS[@]} \
    ${PERF_ARGS[@]} \
    ${MISC_ARGS[@]}
[tab] slurm/dev/mihir/berlin/crowd_pilot/launch_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --mem=100GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/dyn/causal/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/dyn/causal/%x_%j.log
#SBATCH --job-name=sg_lang_server


FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0
[tab] slurm/jobs/mihir/juelich/crowd-pilot/launch_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --mem=100GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/dyn/causal/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/dyn/causal/%x_%j.log
#SBATCH --job-name=sg_lang_server


FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

python3 -m sglang.launch_server --model-path qwen/qwen2.5-0.5b-instruct --host 0.0.0.0
[tab] slurm/dev/mihir/berlin/crowd_pilot/evaluate/launch_judge_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --job-name=run_judge
#SBATCH --qos=low

module load CUDA/12.8
FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1 

# model_id="qwen/qwen2.5-0.5b-instruct"
model_id="Qwen/Qwen3-30B-A3B"
# model_id="Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa_bak/launch_judge_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --job-name=run_judge
#SBATCH --qos=low

module load CUDA/12.8
FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1 

# model_id="qwen/qwen2.5-0.5b-instruct"
model_id="Qwen/Qwen3-30B-A3B"
# model_id="Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[tab] slurm/jobs/mihir/juelich/crowd-pilot/launch_judge_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --job-name=run_judge
#SBATCH --qos=low

module load CUDA/12.8
FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1 

# model_id="qwen/qwen2.5-0.5b-instruct"
model_id="Qwen/Qwen3-30B-A3B"
# model_id="Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[tab] slurm/dev/alfred/juwels/evals/evaluate/host_qwen_30B_non_thinking.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=2:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=host_qwen3_30B_non_thinking

FLASHINFER_CACHE_DIR=$PROJECT_envcomp/$USER/.cache/flashinfer

source .venv/bin/activate


module load Stages/2026 CUDA/13

model_id="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-30B-A3B-Instruct-2507/snapshots/0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe"

python3 -m sglang.launch_server \
    --model-path $model_id \
    --host 0.0.0.0 \
    --port 30000 \
    --tp-size 2
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa_bak/sbatch_files/sbatch_qwen_1.7b_lora_bs8.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0000-0099]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=qwen_1.7b_lora_bs_8



trap 'echo "Job killed by signal"' USR1

cat $0

module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

source .venv/bin/activate

srun sh slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_lora_bs_8.sh
[tab] slurm/jobs/mihir/juelich/crowd-pilot/run_sqwen3-0.6b-sft-torch-crowd-code.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch-crowd-code


trap 'echo "Job killed by signal"' USR1

cat $0

module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

source .venv/bin/activate

# srun sh slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_lora_bs_16.sh 
# srun sh slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_8b_lora_bs_16.sh
srun sh slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_4b_no_lora_bs_16.sh