p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] scripts/run-sft-torchrun.sh (shellscript)
#!/bin/bash
#
# Ray-free SFT Training Script
#
export PYTHONUNBUFFERED=1
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
# FIXME(f.srambical): this is hardcoded for now
GPUS_PER_NODE=${SLURM_GPUS_ON_NODE}
NUM_NODES=${SLURM_JOB_NUM_NODES}
NODE_RANK=${SLURM_NODEID}
MASTER_ADDR=${MASTER_ADDR:-$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)}

NVLINK_COUNT=$(nvidia-smi | grep -o "NVLink" | wc -l)
if [ "$NVLINK_COUNT" -gt 0 ]; then
    HAS_NVLINK=1
else
    HAS_NVLINK=0
fi
echo "HAS_NVLINK: $HAS_NVLINK (detected $NVLINK_COUNT NVLink references)"

export NCCL_DEBUG=INFO
export TORCH_DISTRIBUTED_DEBUG=INFO

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"

RUN_ID=${RUN_ID:-"run_$(date +%Y%m%d_%H%M%S)"}
LOAD_PATH="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B"
SAVE_PATH="/fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/${RUN_ID}/checkpoints"

CKPT_ARGS=(
   --hf-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B
   --load ${LOAD_PATH}
   --ref-load /fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B
   --save ${SAVE_PATH}
   --save-interval 200
)

SFT_ARGS=(
   --rollout-function-path miles.rollout.sft_rollout.generate_rollout
   --prompt-data /fast/project/HFMI_SynergyUnit/tab_model/huggingface/nemo_hf_part_jsonl_4k_tokens.parquet
   --input-key messages
   --apply-chat-template
   --rollout-shuffle
   --num-epoch 3
   --rollout-batch-size 16
   --global-batch-size 16

   --loss-type sft_loss
   --calculate-per-token-loss
   --disable-compute-advantages-and-returns
   --num-rollout 2000
)

LORA_ARGS=(
    # --use-lora
    # --lora-rank 8
    # --lora-alpha 16
    # --lora-dropout 0.0
    # --lora-target-modules q_proj v_proj
)

OPTIMIZER_ARGS=(
   --optimizer adam
   --lr 1e-5
   --lr-decay-style WSD
   --lr-wsd-decay-style linear
   --lr-warmup-iters 100
   --lr-decay-iters 2000
   --lr-wsd-decay-iters 500
   --weight-decay 0.1
   --adam-beta1 0.9
   --adam-beta2 0.98
)

WANDB_ARGS=(
   --use-wandb
   --wandb-project crowd-pilot-miles
   --wandb-team instant-uv
   --wandb-group qwen3-0.6b-sft-torchrun
)

TRAIN_BACKEND_ARGS=(
   --train-backend fsdp
   --update-weight-buffer-size 536870912
   --gradient-checkpointing
   --attn-implementation flash_attention_3
)

PERF_ARGS=(
   --use-dynamic-batch-size
   --max-tokens-per-gpu 9216
)

MISC_ARGS=(
   --rollout-max-context-len 8192
   --rollout-max-prompt-len 8000
   --rollout-max-response-len 8192
   --dump-details /fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/qwen3-600M-fsdp-1116-noref/dump_details
)

torchrun \
    --nproc_per_node=${GPUS_PER_NODE} \
    --nnodes=${NUM_NODES} \
    --node_rank=${NODE_RANK} \
    --master_addr=${MASTER_ADDR} \
    --master_port=${MASTER_PORT:-29500} \
    train_sft.py \
    ${CKPT_ARGS[@]} \
    ${SFT_ARGS[@]} \
    ${LORA_ARGS[@]} \
    ${OPTIMIZER_ARGS[@]} \
    ${WANDB_ARGS[@]} \
    ${TRAIN_BACKEND_ARGS[@]} \
    ${PERF_ARGS[@]} \
    ${MISC_ARGS[@]}
[tab] inspect_model.py (python)
import torch
from transformers import AutoConfig, AutoModelForCausalLM

model_path = "/fast/project/HFMI_SynergyUnit/tab_model/huggingface/Qwen3-0.6B"
config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_config(config, trust_remote_code=True)

classes = set()
for name, module in model.named_modules():
    if "attn" in name.lower():
        classes.add(type(module))

for cls in classes:
    print(cls)
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/miles/test_lora_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --job-name=crowd_pilot_sglang
#SBATCH --mem=400GB
#SBATCH --qos=normal

source /home/franz.srambical/crowd-pilot-serializer-legacy/.venv/bin/activate
module load CUDA/12.8

model_path="Qwen/Qwen3-8b"
#model_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=949-consumed_samples=950.0"
python3 -m sglang.launch_server --model-path $model_path --host 0.0.0.0 --log-requests
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/miles/test_lora_sglang.sh (shellscript)
--lora-checkpoint /fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=949-consumed_samples=950.0
[content] /home/franz.srambical/slurm/dev/franz/berlin/crowd-pilot/miles/test_lora_sglang.sh (shellscript)
python3 -m sglang.launch_server --model-path $model_path --host 0.0.0.0 --log-requests --enable-lora --lora-rank 8 --lora-alpha 16 --lora-dropout 0.0 --lora-target-modules q_proj v_proj --lora-paths /fast/project/HFMI_SynergyUnit/tab_model/huggingface/shared_data/run_20251222_220632/checkpoints/iter_0000800/