p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/iterating_on_stuff/test_full_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm
from lightning.pytorch.loggers import WandbLogger

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"
slurm_job_id = os.environ.get("SLURM_JOB_ID", "local")

NAME = f"qwen3_600m_full_4k_tokens_debug_{slurm_job_id}"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

def recipe_to_dict(recipe, depth=4):
    if depth == 0:
        return dict()
    cfg_dict = dict()
    for k, v in vars(recipe)['__arguments__'].items():
        if isinstance(v, run.Config) or isinstance(v, dict):
            cfg_dict[k] = recipe_to_dict(v, depth-1)
        elif isinstance(v, (int, float, str, bool)):
            cfg_dict[k] = v
        else:
            cfg_dict[k] = str(v)
    return cfg_dict
                    
recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/mihir/logs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=32,
        micro_batch_size=4,
        num_workers=0,
    )

    MAX_STEPS = 11
    VAL_INTERVAL = 2
    NUM_CHECKPOINTS = 10
    recipe.trainer.max_steps = MAX_STEPS
    recipe.log.ckpt.every_n_train_steps = MAX_STEPS // NUM_CHECKPOINTS
    recipe.log.ckpt.save_top_k = NUM_CHECKPOINTS
    recipe.trainer.val_check_interval = VAL_INTERVAL

    cfg_dict = recipe_to_dict(recipe)
    recipe.log.wandb = run.Config(
        WandbLogger,
        name=NAME,
        project="crowd-pilot-nemo",
        save_dir="/fast/project/HFMI_SynergyUnit/mihir/logs/wandb",
        entity="instant-uv",
        config=cfg_dict,
    )

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()