p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/franz/berlin/crowd-pilot/start_sglang_server.sh (shellscript)
#!/usr/bin/env bash

source /home/franz.srambical/crowd-pilot/.venv/bin/activate
module load CUDA/12.6.0

model_id="Qwen/Qwen3-8b"
python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[terminal_command] TERMINAL ()
ls /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name\=0--val_loss\=0.30-step\=999-consumed_samples\=1000.0-last/
[terminal_command] TERMINAL ()
ls -R /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name\=0--val_loss\=0.30-step\=999-consumed_samples\=1000.0-last/
[terminal_command] TERMINAL ()
cat /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name\=0--val_loss\=0.30-step\=999-consumed_samples\=1000.0-last/context/nemo_tokenizer/tokenizer.json
[terminal_command] TERMINAL ()
ls /fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name\=0--val_loss\=0.30-step\=999-consumed_samples\=1000.0-last/weights/
[tab] slurm/dev/franz/berlin/crowd-pilot/qwen_0_6/convert_checkpoint_from_hf.sh (shellscript)
#!/usr/bin/env bash
set -uex

export XLA_PJRT_GPU_HOST_MEMORY_LIMIT_GB=400
cd maxtext
source .venv/bin/activate

export HF_HOME="$TMPDIR/.cache/huggingface"

MODEL_NAME="qwen3-0.6b"
OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_maxtext/Qwen/Qwen3-0.6B"

# Convert using the general conversion framework
python3 -m MaxText.utils.ckpt_conversion.to_maxtext \
  src/MaxText/configs/base.yml \
  model_name=${MODEL_NAME} \
  base_output_directory=${OUTPUT_DIR} \
  use_multimodal=false \
  scan_layers=true \
  run_name=qwen3_0_6b_conversion
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/download_models/download_qwen3_600m_nemo.py (python)
from nemo.collections import llm

if __name__ == "__main__":
  llm.import_ckpt(model=llm.Qwen3Model(llm.Qwen3Config600M()), source='hf://Qwen/Qwen3-0.6B', output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m")
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/export_model_to_hf/export_qwen_0_6b.py (python)
from nemo.collections import llm

if __name__ == "__main__":
  llm.import_ckpt(model=llm.Qwen3Model(llm.Qwen3Config600M()), source='hf://Qwen/Qwen3-0.6B', output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m")
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/export_model_to_hf/export_qwen_0_6b.py (python)
/fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name\=0--val_loss\=0.30-step\=999-consumed_samples\=1000.0-last/weights/
[tab] slurm/utils/alfred/scp_scripts/copy_to_local.sh (shellscript)
#!/bin/bash

dst=/fast/project/HFMI_SynergyUnit/jafar_ws
src=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/checkpoints_alfred

rsync -avzP $src alfred.nguyen@login.haicore.berlin:$dst
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/export_model_to_hf/export_qwen_0_6b.py (python)
  llm.export_ckpt(model=llm.Qwen3Model(llm.Qwen3Config600M()), target='hf', path='/fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/', output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/")
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/export_model_to_hf/export_qwen_0_6b.py (python)
  llm.export_ckpt(
    target='hf',
    path='/fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/',
    output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/",
    load_connector_strict=False,
  )
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/export_model_to_hf/export_qwen_0_6b.py (python)

  llm.export_ckpt(target='hf', path='/fast/project/HFMI_SynergyUnit/tab_model/data/outputs/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/', output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=999-consumed_samples=1000.0-last/")
[content] slurm/jobs/franz/berlin/crowd-pilot/start_sglang_server.sh (shellscript)
model_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/trained_checkpoints_converted_to_hf/qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1/2025-11-23_18-58-56/checkpoints/model_name=0--val_loss=0.30-step=949-consumed_samples=950.0"