p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] check_seq_lens.py (python)
import json
from transformers import AutoTokenizer
import numpy as np

model_path = "/p/scratch/envcomp/mihir/huggingface/Qwen3-4B"
val_path = "/p/scratch/envcomp/mihir/huggingface/dataset_05_01_2026/qwen/miles_hf_part_jsonl_4k_tokens/validation.jsonl"

print(f"Loading tokenizer from {model_path}...")
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

lengths = []
exceed_4096 = 0
exceed_4999 = 0

print(f"Reading validation file {val_path}...")
with open(val_path, 'r') as f:
    for line in f:
        data = json.loads(line)
        messages = data.get("messages")
        if messages:
            # Apply chat template
            token_ids = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=False)
            length = len(token_ids)
            lengths.append(length)
            if length > 4096:
                exceed_4096 += 1
            if length > 4999:
                exceed_4999 += 1

lengths = np.array(lengths)
print("
--- Statistics ---")
print(f"Total samples: {len(lengths)}")
print(f"Mean length: {np.mean(lengths):.2f}")
print(f"Median length: {np.median(lengths)}")
print(f"Max length: {np.max(lengths)}")
print(f"Min length: {np.min(lengths)}")
print(f"Samples > 4096: {exceed_4096} ({100 * exceed_4096 / len(lengths):.2f}%)")
print(f"Samples > 4999: {exceed_4999} ({100 * exceed_4999 / len(lengths):.2f}%)")

# Percentiles
for p in [10, 25, 50, 75, 90, 95, 99]:
    print(f"{p}th percentile: {np.percentile(lengths, p)}")
[tab] slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_4k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 4096 \
--tokenizer="Qwen/Qwen3-8B"
[tab] slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_8k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_4k_tokens/" \
--max-tokens-per-conversation 8192 \
--tokenizer="Qwen/Qwen3-8B"
[tab] slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_16k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_16k_tokens/" \
--max-tokens-per-conversation 16384 \
--tokenizer="Qwen/Qwen3-8B"
[tab] slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_32k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_32k_tokens/" \
--max-tokens-per-conversation 32768 \
--tokenizer="Qwen/Qwen3-8B"
[tab] slurm/dev/franz/berlin/crowd-pilot/crowd_pilot_serializer/serialize_128k_qwen.sh (shellscript)
./target/release/crowd-pilot-serialize \
--csv-root="/fast/project/HFMI_SynergyUnit/tab_model/data/hf_part_csv/" \
--output-dir="/fast/project/HFMI_SynergyUnit/tab_model/data/qwen/miles_hf_part_jsonl_128k_tokens/" \
--max-tokens-per-conversation 131072 \
--tokenizer="Qwen/Qwen3-8B"
[content] run_glm_2_nodes.sh (shellscript)
#!/bin/bash -l

#SBATCH -o SLURM_Logs/%x_%j_master.out
#SBATCH -e SLURM_Logs/%x_%j_master.err
#SBATCH -D ./
#SBATCH -J Llama-405B-Online-Inference-TP16-SGL

#SBATCH --nodes=2
#SBATCH --ntasks=2
#SBATCH --ntasks-per-node=1  # Ensure 1 task per node
#SBATCH --cpus-per-task=18
#SBATCH --mem=224GB
#SBATCH --partition="lmsys.org"
#SBATCH --gres=gpu:8
#SBATCH --time=12:00:00

echo "[INFO] Activating environment on node $SLURM_PROCID"
if ! source ENV_FOLDER/bin/activate; then
    echo "[ERROR] Failed to activate environment" >&2
    exit 1
fi

# Define parameters
model=MODEL_PATH
tp_size=16

echo "[INFO] Running inference"
echo "[INFO] Model: $model"
echo "[INFO] TP Size: $tp_size"

# Set NCCL initialization address using the hostname of the head node
HEAD_NODE=$(scontrol show hostname "$SLURM_NODELIST" | head -n 1)
NCCL_INIT_ADDR="${HEAD_NODE}:8000"
echo "[INFO] NCCL_INIT_ADDR: $NCCL_INIT_ADDR"

# Launch the model server on each node using SLURM
srun --ntasks=2 --nodes=2 --output="SLURM_Logs/%x_%j_node$SLURM_NODEID.out" \
    --error="SLURM_Logs/%x_%j_node$SLURM_NODEID.err" \
    python3 -m sglang.launch_server \
    --model-path "$model" \
    --grammar-backend "xgrammar" \
    --tp "$tp_size" \
    --dist-init-addr "$NCCL_INIT_ADDR" \
    --nnodes 2 \
    --node-rank "$SLURM_NODEID" &

# Wait for the NCCL server to be ready on port 30000
while ! nc -z "$HEAD_NODE" 30000; do
    sleep 1
    echo "[INFO] Waiting for $HEAD_NODE:30000 to accept connections"
done

echo "[INFO] $HEAD_NODE:30000 is ready to accept connections"

# Keep the script running until the SLURM job times out
wait
[content] run_glm_2_nodes.sh (shellscript)
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=02:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --job-name=eval_miles_run