p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/berlin/crowd-pilot/run_sqwen3-0.6b-sft.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=10
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft

cat $0

source ~/.bashrc

micromamba activate miles

sh scripts/run-qwen3-0.6B-base-sft.sh
[tab] slurm/jobs/mihir/berlin/crowd-pilot/run_sqwen3-0.6b-sft copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=10
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft

cat $0

source ~/.bashrc

micromamba activate miles

sh scripts/run-qwen3-0.6B-base-sft.sh
[tab] slurm/jobs/mihir/berlin/crowd-pilot/run_sqwen3-0.6b-sft_dev.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=10
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:8
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft

cat $0

source ~/.bashrc

micromamba activate miles

sh scripts/run-qwen3-0.6B-base-sft.sh
[content] scripts/run-qwen3-0.6B-base-sft_dev.sh (shellscript)
    
#!/bin/bash

# --- CLEANUP PREVIOUS RUNS ---
pkill -9 sglang
sleep 3
ray stop --force
pkill -9 ray
pkill -9 python
sleep 3
# Double tap to be sure
pkill -9 ray
pkill -9 python

set -ex

# --- 1. DYNAMIC HOST IP DETECTION (CRITICAL FOR SLURM) ---
# Don't hardcode 10.86.2.50. Get the actual IP of the current node.
export HEAD_NODE_IP=$(hostname -I | awk '{print $1}')
echo "Detected Head Node IP: ${HEAD_NODE_IP}"

# --- 2. PROXY CONFIGURATION ---
# Ensure local traffic doesn't go through a corporate proxy
export no_proxy="${HEAD_NODE_IP},localhost,127.0.0.1,0.0.0.0"
export NO_PROXY="${HEAD_NODE_IP},localhost,127.0.0.1,0.0.0.0"

export PYTHONBUFFERED=16

# Check NVLink
NVLINK_COUNT=$(nvidia-smi topo -m 2>/dev/null | grep -o 'NV[0-9][0-9]*' | wc -l)
if [ "$NVLINK_COUNT" -gt 0 ]; then
    HAS_NVLINK=1
else
    HAS_NVLINK=0
fi

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
source "${SCRIPT_DIR}/models/qwen3-0.6B.sh"
[content] scripts/run-qwen3-0.6B-base-sft_dev.sh (shellscript)
# --- 3. START RAY HEAD ---
# Use the dynamic IP detected above
ray start --head \
    --node-ip-address=${HEAD_NODE_IP} \
    --num-gpus 8 \
    --disable-usage-stats \
    --dashboard-host=0.0.0.0 \
    --dashboard-port=8265 \
    --port=6379

echo "Ray started. Waiting for Dashboard to be ready..."

# --- 4. WAIT FOR DASHBOARD (FIX FOR 504 ERROR) ---
# Loop until the dashboard port accepts connections
for i in {1..30}; do
    if curl -s "http://${HEAD_NODE_IP}:8265" > /dev/null; then
        echo "Dashboard is up!"
        break
    fi
    echo "Waiting for Ray Dashboard..."
    sleep 2
done
# Add a small safety buffer
sleep 5

# --- 5. SUBMIT JOB ---

# Build runtime env
RUNTIME_ENV_JSON="{
  \"env_vars\": {
    \"PYTHONPATH\": \"/fast/project/HFMI_SynergyUnit/tab_model/Megatron-LM/\",
    \"CUDA_DEVICE_MAX_CONNECTIONS\": \"1\",
    \"NCCL_NVLS_ENABLE\": \"${HAS_NVLINK}\",
    \"PYTORCH_CUDA_ALLOC_CONF\": \"expandable_segments:True\"
  }
}"

# Submit using the dynamic IP
ray job submit --address="http://${HEAD_NODE_IP}:8265" \
   --runtime-env-json="${RUNTIME_ENV_JSON}" \
   -- python3 train_async.py \
   --actor-num-nodes 1 \
   --actor-num-gpus-per-node 2 \
   ${MODEL_ARGS[@]} \
   ${CKPT_ARGS[@]} \
   ${SFT_ARGS[@]} \
   ${OPTIMIZER_ARGS[@]} \
   ${WANDB_ARGS[@]} \
   ${PERF_ARGS[@]} \
   ${EVAL_ARGS[@]} \
   ${MISC_ARGS[@]}
[tab] slurm/jobs/mihir/berlin/crowd-pilot/build_slime.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --gres=gpu:1
#SBATCH --cpus-per-task=16
#SBATCH --output=/home/mihir.mahajan/logs/build_slime/%x_%j.log
#SBATCH --error=/home/mihir.mahajan/logs/build_slime/%x_%j.log
#SBATCH --job-name=build_slime
#SBATCH --mem=500G

source ~/.bashrc

echo "Activating miles environment"
micromamba activate miles

ARCH=$(python -c "import torch; print('.'.join(map(str, torch.cuda.get_device_capability())))")
export TORCH_CUDA_ARCH_LIST=$ARCH

export PATH="$CONDA_PREFIX/bin:$PATH"

export CUDA_HOME="$CONDA_PREFIX"
echo "CUDA_HOME: $CUDA_HOME"

export BASE_DIR=${BASE_DIR:-"/fast/project/HFMI_SynergyUnit/mihir/"}
echo "BASE_DIR: $BASE_DIR"
cd $BASE_DIR

cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
echo "sglang checked out"

echo "Installing flash-attn"
export MAX_JOBS=8
pip -v install flash-attn==2.7.4.post1 --no-build-isolation --no-cache-dir
echo "flash-attn installed"
[tab] build_conda.sh (shellscript)
#!/bin/bash

# set -ex

# create conda
# yes '' | "${SHELL}" <(curl -L micro.mamba.pm/install.sh)
# export PS1=tmp
# mkdir -p /fast/project/HFMI_SynergyUnit/mihir/.cargo/
# touch /fast/project/HFMI_SynergyUnit/mihir/.cargo/env
source ~/.bashrc

#micromamba create -n miles python=3.12 pip -c conda-forge -y
micromamba activate miles
export CUDA_HOME="$CONDA_PREFIX"

export BASE_DIR=${BASE_DIR:-"/fast/project/HFMI_SynergyUnit/mihir/"}
cd $BASE_DIR

# install cuda 12.9 as it's the default cuda version for torch
# micromamba install -n miles cuda cuda-nvtx cuda-nvtx-dev nccl -c nvidia/label/cuda-12.9.1 -y
# micromamba install -n miles -c conda-forge cudnn -y

# prevent installing cuda 13.0 for sglang
# pip install cuda-python==12.9.1
# pip install torch==2.8.0 torchvision==0.23.0 torchaudio==2.8.0 --index-url https://download.pytorch.org/whl/cu129

# install sglang
# git clone https://github.com/sgl-project/sglang.git
cd sglang
git checkout 303cc957e62384044dfa8e52d7d8af8abe12f0ac
# Install the python packages
# pip install -e "python[all]"


# pip install cmake ninja

ARCH=$(python -c "import torch; print('.'.join(map(str, torch.cuda.get_device_capability())))")
export TORCH_CUDA_ARCH_LIST=$ARCH

# flash attn
# the newest version megatron supports is v2.7.4.post1
# MAX_JOBS=64 pip -v install flash-attn==2.7.4.post1 --no-build-isolation
# pip install https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.3.18/flash_attn-2.7.4%2Bcu128torch2.8-cp312-cp312-linux_x86_64.whl

# pip install git+https://github.com/ISEEKYAN/mbridge.git@89eb10887887bc74853f89a4de258c0702932a1c --no-deps
# pip install --no-build-isolation "transformer_engine[pytorch]==2.8.0" --no-cache-dir
# pip install flash-linear-attention==0.4.0
# NVCC_APPEND_FLAGS="--threads 4" \
  # pip -v install --disable-pip-version-check --no-cache-dir \
  # --no-build-isolation \
  # --config-settings "--build-option=--cpp_ext --cuda_ext --parallel 8" git+https://github.com/NVIDIA/apex.git@10417aceddd7d5d05d7cbf7b0fc2daad1105f8b4

git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
    cd Megatron-LM && git checkout ${MEGATRON_COMMIT} && \
    pip install -e .

pip install git+https://github.com/fzyzcjy/torch_memory_saver.git@9b8b788fdeb9c2ee528183214cef65a99b71e7d5 --no-cache-dir --force-reinstall
pip install git+https://github.com/fzyzcjy/Megatron-Bridge.git@dev_rl --no-build-isolation
pip install nvidia-modelopt[torch]>=0.37.0 --no-build-isolation

# megatron
cd $BASE_DIR
git clone https://github.com/NVIDIA/Megatron-LM.git --recursive && \
  cd Megatron-LM/ && git checkout core_v0.14.0 && \
  pip install -e .

# install miles and apply patches

# if miles does not exist locally, clone it
if [ ! -d "$BASE_DIR/miles" ]; then
  cd $BASE_DIR
  git clone  https://github.com/radixark/miles.git
  cd miles/
  export MILES_DIR=$BASE_DIR/miles
  pip install -e .
else
  export MILES_DIR=$BASE_DIR/
  pip install -e .
fi

# apply patch
cd $BASE_DIR/sglang
git apply $MILES_DIR/docker/patch/v0.5.5.post1/sglang.patch
cd $BASE_DIR/Megatron-LM
git apply $MILES_DIR/docker/patch/v0.5.5.post1/megatron.patch
[tab] /fast/home/mihir.mahajan/Projects/miles/miles/backends/megatron_utils/checkpoint.py (python)
import logging
import os
from pathlib import Path

# TODO: may need to copy those 2 functions and do refactoring.
from megatron.training.checkpointing import load_checkpoint as _load_checkpoint_megatron
from megatron.training.checkpointing import save_checkpoint
from megatron.training.global_vars import get_args
from miles.utils import megatron_bridge_utils

logger = logging.getLogger(__name__)

__all__ = ["save_checkpoint"]


def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, checkpointing_context, skip_load_to_model_and_opt):
    # ref: how megatron `load_checkpoint` gets directory
    args = get_args()
    load_path = args.load

    assert Path(load_path).exists() and _is_dir_nonempty(
        load_path
    ), f"{args.load=} does not exist or is an empty directory. Did you specify the wrong folder?"

    if _is_megatron_checkpoint(load_path):
        return _load_checkpoint_megatron(
            ddp_model=ddp_model,
            optimizer=optimizer,
            opt_param_scheduler=opt_param_scheduler,
            checkpointing_context=checkpointing_context,
            skip_load_to_model_and_opt=skip_load_to_model_and_opt,
        )
    else:
        return _load_checkpoint_hf(
            ddp_model=ddp_model,
            optimizer=optimizer,
            args=args,
            load_path=load_path,
        )


def _is_megatron_checkpoint(path: str | Path) -> bool:
    return (Path(path) / "latest_checkpointed_iteration.txt").is_file()


def _load_checkpoint_hf(ddp_model, optimizer, args, load_path: str):
    from megatron.bridge import AutoBridge
    import miles_plugins.megatron_bridge  # noqa: F401

    logger.info(f"Load checkpoint from HuggingFace model into Megatron (path={load_path})")
    bridge = AutoBridge.from_hf_pretrained(load_path, trust_remote_code=True)

    with megatron_bridge_utils.patch_megatron_model(ddp_model):
        bridge.load_hf_weights(ddp_model)

    # Copied from Megatron-core :: load_checkpoint (with simplifications)
    if (args.fp16 or args.bf16) and optimizer is not None:
        assert not args.load_main_params_from_ckpt
        optimizer.reload_model_params()

    # We can see `successfully loaded checkpoint from ... [ t 1/2, p 1/1 ] at iteration 0`
    # when loading Megatron, thus it is 0
    iteration = 0
    num_floating_point_operations_so_far = 0
    return iteration, num_floating_point_operations_so_far


def _is_dir_nonempty(path):
    with os.scandir(path) as it:
        return any(it)
[content] slurm/jobs/mihir/berlin/crowd-pilot/convert_dataset.py (python)
from datasets import load_dataset

ds = load_dataset("teknium/OpenHermes-2.5")["train"]

def convert(sample):
    conversations = sample["conversations"]

    def convert_role(role):
        if role == "human":
            return "user"
        elif role == "gpt":
            return "assistant"
        elif role == "system":
            return "system"
        else:
            raise ValueError(f"Unknown role: {role}")

    messages = [
        {
            "role": convert_role(turn["from"]),
            "content": turn["value"],
        }
        for turn in conversations
    ]

    return {"messages": messages}

ds = ds.map(convert)
ds.to_parquet("/fast/project/HFMI_SynergyUnit/tab_model/huggingface/openhermes2_5.parquet")