p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune.py (python)
import nemo_run as run
from nemo.lightning.pytorch.callbacks import MemoryProfileCallback, NsysCallback
from nemo.collections import llm

NAME = "qwen3_30b_a3b_lora_finetune_profiling"

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"test_output",
    num_nodes=1,
    num_gpus_per_node=8,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=True,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    recipe.data.delete_raw = False
    # FIXME (f.srambical): remove this after profiling
    recipe.trainer.max_steps = 16
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    nsys_callback = NsysCallback(
        start_step=10,
        end_step=15,
    )
    recipe.trainer.callbacks.append(nsys_callback)
    recipe.trainer.callbacks.append(MemoryProfileCallback(dir="test_output/memory_traces", ranks=[0, 1]))

    run.run(recipe, direct=True)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_full_finetune.py (python)
import nemo_run as run
from nemo.collections import llm

NAME = "qwen3_30b_a3b_full_finetune"

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"test_output",
    num_nodes=1,
    num_gpus_per_node=8,
    peft_scheme=None,
    packed_sequence=True,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    recipe.data.delete_raw = False
    recipe.data.micro_batch_size = 1
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()