p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen-coder-small/convert_checkpoint.sh (shellscript)
#!/usr/bin/env bash
set -uex

export XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB=400
cd maxtext
source .venv/bin/activate

MODEL_NAME="Qwen/Qwen3-Coder-30B-A3B-Instruct"
HF_MODEL_PATH="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_hf/${MODEL_NAME}"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/${MODEL_NAME}"

python3 -m MaxText.convert_qwen3_moe\
  --base_model_path ${HF_MODEL_PATH}\
  --maxtext_model_path ${OUTPUT_DIR}\
  --model_size qwen3-30b-a3b \
  --simulated_cpu_devices_count=2
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/download_checkpoint.sh (shellscript)
#!/usr/bin/env bash
set -uex

source .venv/bin/activate

MODEL_NAME="Qwen/Qwen3-Coder-30B-A3B-Instruct"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_hf/${MODEL_NAME}"

HF_HUB_ENABLE_HF_TRANSFER=1 hf download ${MODEL_NAME} --local-dir ${OUTPUT_DIR}
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
#!/usr/bin/env bash
set -uex

export XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB=400
cd maxtext
source .venv/bin/activate

MODEL_NAME="Qwen/Qwen3-Coder-30B-A3B-Instruct"
HF_MODEL_PATH="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_hf/${MODEL_NAME}"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/${MODEL_NAME}"

python3 -m MaxText.convert_qwen3_moe\
  --base_model_path ${HF_MODEL_PATH}\
  --maxtext_model_path ${OUTPUT_DIR}\
  --model_size qwen3-30b-a3b \
  --simulated_cpu_devices_count=2
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/train_10_epochs.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_maxtext_array_record_10_epochs
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/outputs/
GRAIN_TRAIN_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  num_epoch=10 \
  skip_jax_distributed_system=True
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/train.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:8
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_qwen3-30b-a3b_dev
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/outputs/
PRETRAINED_CHECKPOINT_PATH=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-Coder-30B-A3B-Instruct/
GRAIN_TRAIN_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  per_device_batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  skip_jax_distributed_system=True
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen-coder-small/train_10_epochs.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_maxtext_array_record_10_epochs
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/outputs/
GRAIN_TRAIN_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  per_device_batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  num_epoch=10 \
  skip_jax_distributed_system=True
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/train_10_epochs.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_maxtext_array_record_10_epochs
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/outputs/
GRAIN_TRAIN_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  per_device_batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  num_epoch=10 \
  skip_jax_distributed_system=True
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/train.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:8
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_qwen3-30b-a3b_dev
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/outputs/
PRETRAINED_CHECKPOINT_PATH=/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-Coder-30B-A3B-Instruct/
GRAIN_TRAIN_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  per_device_batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  skip_jax_distributed_system=True
[tab] slurm/dev/franz/helmholtz_cluster/crowd-pilot/qwen-coder-small/train_10_epochs.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --error=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/logs/franz/crowd-pilot/maxtext/%x_%j.log
#SBATCH --job-name=crowd-pilot_maxtext_array_record_10_epochs
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# Log the sbatch script
cat $0

cd maxtext
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID
BASE_OUTPUT_DIRECTORY=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/data/crowd-pilot/outputs/
PRETRAINED_CHECKPOINT_PATH=/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-Coder-30B-A3B-Instruct/
GRAIN_TRAIN_FILES="/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/data/crowd-pilot/crowd-code-0.1/array_record/train_*.array_record"
GRAIN_EVAL_FILES="/lustre/groups/haicu/workspace/alfred.nguyen/jafar_worksapce/data/crowd-pilot/crowd-code-0.1/array_record/val_*.array_record"

env | grep SLURM

# FIXME(f.srambical): change the batch size to the largest one that can fit in accelerator memory
python3 -m MaxText.train \
  MaxText/configs/base.yml \
  run_name=${job_name}_${slurm_job_id} \
  base_output_directory=$BASE_OUTPUT_DIRECTORY \
  load_parameters_path=$PRETRAINED_CHECKPOINT_PATH/0/items/ \
  model_name=qwen3-30b-a3b \
  max_target_length=8192 \
  eval_interval=500 \
  steps=25000 \
  per_device_batch_size=1 \
  tokenizer_type=huggingface \
  tokenizer_path=src/MaxText/assets/qwen3-tokenizer \
  dataset_type=grain \
  grain_file_type=arrayrecord \
  grain_train_files=$GRAIN_TRAIN_FILES \
  grain_eval_files=$GRAIN_EVAL_FILES \
  num_epoch=10 \
  skip_jax_distributed_system=True
[content] maxtext/src/MaxText/convert_qwen3_moe.py (python)

        "num_hidden_layers": 24,
        "num_attention_heads": 16,
        "num_key_value_heads": 4,
        "hidden_size": 1024,
        "head_dim": 128,
        "num_experts": 128,
        "moe_intermediate_size": 384,
    },
[content] maxtext/src/MaxText/convert_qwen3_moe.py (python)
    "qwen3-0.6b": {
        "num_hidden_layers": 28,
        "num_attention_heads": 16,
        "num_key_value_heads": 8,
        "hidden_size": 1024,
        "head_dim": 128,
        "num_experts": 128,
        "moe_intermediate_size": 384,
    },
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
# Set Hugging Face token (required for downloading models)
export HF_TOKEN=${HF_TOKEN:-"your_hf_token_here"}

MODEL_NAME="qwen3-0.6b"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-0.6B"

# Convert using the general conversion framework
python3 -m MaxText.utils.ckpt_conversion.to_maxtext \
  src/MaxText/configs/base.yml \
  model_name=${MODEL_NAME} \
  base_output_directory=${OUTPUT_DIR} \
  hf_access_token=${HF_TOKEN} \
  use_multimodal=false \
  scan_layers=false \
  run_name=qwen3_0_6b_conversion
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
MODEL_NAME="Qwen/Qwen3-0.6B"
HF_MODEL_PATH="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_hf/${MODEL_NAME}"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/${MODEL_NAME}"
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
# Set Hugging Face token (required for downloading models)
export HF_TOKEN=${HF_TOKEN:-"your_hf_token_here"}

MODEL_NAME="qwen3-0.6b"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-0.6B"

# Convert using the general conversion framework
python3 -m MaxText.utils.ckpt_conversion.to_maxtext \
  src/MaxText/configs/base.yml \
  model_name=${MODEL_NAME} \
  base_output_directory=${OUTPUT_DIR} \
  hf_access_token=${HF_TOKEN} \
  use_multimodal=false \
  scan_layers=true \
  run_name=qwen3_0_6b_conversion
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
# Set Hugging Face cache to scratch directory to avoid home directory space issues
export HF_HOME="/fast/project/HFMI_SynergyUnit/jafar_ws/.cache/huggingface"
export TRANSFORMERS_CACHE="/fast/project/HFMI_SynergyUnit/jafar_ws/.cache/huggingface/hub"

# Set Hugging Face token (required for downloading models)
export HF_TOKEN=${HF_TOKEN:-"your_hf_token_here"}

MODEL_NAME="qwen3-0.6b"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-0.6B"

# Convert using the general conversion framework
python3 -m MaxText.utils.ckpt_conversion.to_maxtext \
  src/MaxText/configs/base.yml \
  model_name=${MODEL_NAME} \
  base_output_directory=${OUTPUT_DIR} \
  hf_access_token=${HF_TOKEN} \
  use_multimodal=false \
  scan_layers=true \
  run_name=qwen3_0_6b_conversion
[content] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint.sh (shellscript)
#!/usr/bin/env bash
set -uex

export XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB=400
cd maxtext
source .venv/bin/activate

export HF_HOME="$TMPDIR/.cache/huggingface"
export TRANSFORMERS_CACHE="$TMPDIR/.cache/huggingface/hub"


MODEL_NAME="qwen3-0.6b"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-0.6B"

# Convert using the general conversion framework
python3 -m MaxText.utils.ckpt_conversion.to_maxtext \
  src/MaxText/configs/base.yml \
  model_name=${MODEL_NAME} \
  base_output_directory=${OUTPUT_DIR} \
  use_multimodal=false \
  scan_layers=true \
  run_name=qwen3_0_6b_conversion