p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_30b2a/test_lora_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs",
    num_nodes=1,
    num_gpus_per_node=2,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=64,
        micro_batch_size=8,
        num_workers=0,
    )

    # Enable activation checkpointing to fit 30B model on two H100s
    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1
    recipe.trainer.strategy.ckpt_async_save = False 

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_128"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=128,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sbatch (shellscript)
--seq_length 4096 \
    --global_batch_size 128 \
    --micro_batch_size 1 \
    --num_nodes 1 \
    --num_gpus_per_node 1 \
    --peft_scheme none \
    --packed_sequence false \
    --ckpt_dir /fast/project/HFMI_SynergyUnit/mihir/ckpts/nemo/qwen3_600m \
    --data_dir /fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/ \
    --restore_ckpt_dir /fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/ \
    --tensor_model_parallel_size 1 \
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/finetune.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/mihir/ckpts/nemo/qwen3_600m_dev"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name qwen3_600m_finetune_4k_dev \
    --model_name qwen3_600m \
    --seq_length 4096 \
    --global_batch_size 128 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --packed_sequence false \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR \
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .nemo_venv/bin/activate

python3 "$1"
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=8
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/tab_model/logs/franz/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .nemo_venv/bin/activate

python3 "$1"
[content] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/mihir/ckpts/nemo/qwen3_600m_dev"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name qwen3_600m_finetune_4k_dev \
    --model_name qwen3_600m \
    --seq_length 4096 \
    --global_batch_size 128 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --packed_sequence false \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR
[tab] slurm/jobs/mihir/berlin/crowd-pilot/finetune_lora_4k.sbatch (shellscript)
#!/bin/bash
#SBATCH --job-name=finetune_lora_4k
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/finetune_lora_4k_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/finetune_lora_4k_%j.log
#SBATCH --time=02:00:00
#SBATCH --gres=gpu:2
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=5
#SBATCH --mem=100G

# Activate your environment if needed
source .uv_venv/bin/activate

# Run the finetuning script
srun python slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py
[tab] pyproject.toml (plaintext)
[project]
name = "crowd-pilot"
description = "Teaching language models to code like humans."
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
    "datasets>=2.19.0",
    "pandas>=2.2.2",
    "pyarrow>=15.0.2",
    "tqdm>=4.66.4",
    "regex>=2024.5.15",
    "array-record>=0.8.1",
    "tensorflow>=2.20.0",
    "hf-transfer>=0.1.9",
    "tyro>=0.9.35",
]

[project.urls]
Repository = "https://github.com/p-doom/crowd-pilot"
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/install_nemo.sh (shellscript)
uv venv .nemo_venv --python 3.10.12 --seed
source .nemo_venv/bin/activate

pip install torch==2.8.0 torchvision torchaudio 
pip install nemo_toolkit['all']==2.5.0
pip install --no-build-isolation transformer_engine[pytorch]==2.7.0
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/mihir/ckpts/nemo/qwen3_600m_dev"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name qwen3_600m_finetune_4k_dev \
    --model_name qwen3_600m \
    --seq_length 4096 \
    --global_batch_size 128 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --no_packed_sequence \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune_32k.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/mihir/ckpts/nemo/qwen3_600m_dev"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name qwen3_600m_finetune_4k_dev \
    --model_name qwen3_600m \
    --seq_length 4096 \
    --global_batch_size 128 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --no_packed_sequence \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_32k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_32k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=32,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[terminal_command] TERMINAL ()
tensorboard --logdir /fast/project/HFMI_SynergyUnit/tab_model/mihir/logs/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/tensorboard/default.py:30: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.  import pkg_resourcesTensorFlow installation not found - running with reduced feature set.NOTE: Using experimental fast data loading logic. To disable, pass    "--load_fast=false" and report issues on GitHub. More details:    https://github.com/tensorflow/tensorboard/issues/4784Serving TensorBoard on localhost; to expose to the network, use a proxy or pass --bind_allTensorBoard 2.20.0 at http://localhost:6007/ (Press CTRL+C to quit)^C
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune_32k copy.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune_32k
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name "qwen3_600m_finetune_32k_dev_${SLURM_JOB_ID}" \
    --model_name qwen3_600m \
    --seq_length 32768 \
    --global_batch_size 32 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --no_packed_sequence \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/finetune_16k.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/nemo/%x_%j.log
#SBATCH --job-name=qwen3_600m_finetune_32k
#SBATCH --mem=400GB
#SBATCH --qos=low

# Log the sbatch script
cat $0

source .uv_venv/bin/activate

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs"
mkdir -p $OUTPUT_DIR
DATASET_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/"
RESTORE_CKPT_DIR="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"

python slurm/dev/mihir/berlin/crowd_pilot/nemo/fine_tune.py \
    --name "qwen3_600m_finetune_32k_dev_${SLURM_JOB_ID}" \
    --model_name qwen3_600m \
    --seq_length 32768 \
    --global_batch_size 32 \
    --micro_batch_size 1 \
    --peft_scheme none \
    --no_packed_sequence \
    --ckpt_dir $OUTPUT_DIR \
    --data_dir $DATASET_DIR \
    --restore_ckpt_dir $RESTORE_CKPT_DIR
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_16k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_16k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=1,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_8k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_8k_tokens_batch_size_32"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=8192,
        global_batch_size=32,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_128k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_128k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_128k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=131072,
        global_batch_size=1,
        micro_batch_size=1,
    )
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()