p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
import nemo_run as run
from nemo.collections import llm

NAME = "qwen3_30b_a3b"

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"test_output",
    num_nodes=1,
    num_gpus_per_node=8,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=True,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
        "LD_PRELOAD": "/fast/home/franz.srambical/crowd-pilot/.nemo/lib/python3.10/site-packages/nvidia/cuda_runtime/lib/libcudart.so.12",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    recipe.data.delete_raw = False
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[content] crowd-pilot/temp.out (plaintext)
    296153:	find library=libcrypt.so.1 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libcrypt.so.1
    296153:	
    296153:	find library=libpthread.so.0 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libpthread.so.0
    296153:	
    296153:	find library=libdl.so.2 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libdl.so.2
    296153:	
    296153:	find library=libutil.so.1 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libutil.so.1
    296153:	
    296153:	find library=libm.so.6 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libm.so.6
    296153:	
    296153:	find library=librt.so.1 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/librt.so.1
    296153:	
    296153:	find library=libc.so.6 [0]; searching
    296153:	 search cache=/etc/ld.so.cache
    296153:	  trying file=/lib64/libc.so.6
    296153:	
    296153:	
    296153:	calling init: /lib64/ld-linux-x86-64.so.2
    296153:	
    296153:	
    296153:	calling init: /lib64/libc.so.6
    296153:	
    296153:	
    296153:	calling init: /lib64/librt.so.1
    296153:	
    296153:	
    296153:	calling init: /lib64/libm.so.6
    296153:	
    296153:	
    296153:	calling init: /lib64/libutil.so.1
    296153:	
    296153:	
    296153:	calling init: /lib64/libdl.so.2
    296153:	
    296153:	
    296153:	calling init: /lib64/libpthread.so.0
    296153:	
    296153:	
    296153:	calling init: /lib64/libcrypt.so.1
    296153:	
    296153:	
    296153:	calling init: /fast/home/franz.srambical/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/bin/../lib/libpython3.10.so.1.0
    296153:	
    296153:	
    296153:	initialize program: python
    296153:	
    296153:	
    296153:	transferring control: python
    296153:	
  File "<string>", line 1
    import nemo_run as run \from nemo.collections import llm
                            ^
SyntaxError: unexpected character after line continuation character
    296153:	
    296153:	calling fini: python [0]
    296153:	
    296153:	
    296153:	calling fini: /fast/home/franz.srambical/.local/share/uv/python/cpython-3.10.12-linux-x86_64-gnu/bin/../lib/libpython3.10.so.1.0 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/libcrypt.so.1 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/libpthread.so.0 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/libdl.so.2 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/libutil.so.1 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/libm.so.6 [0]
    296153:	
    296153:	
    296153:	calling fini: /lib64/librt.so.1 [0]
    296153: