p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/iterating_on_stuff/test_full_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm
from lightning.pytorch.loggers import WandbLogger

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"
slurm_job_id = os.environ.get("SLURM_JOB_ID", "local")

NAME = f"qwen3_600m_full_4k_tokens_{slurm_job_id}"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

def recipe_to_dict(recipe, depth=4):
    if depth == 0:
        return dict()
    cfg_dict = dict()
    for k, v in vars(recipe)['__arguments__'].items():
        if isinstance(v, run.Config) or isinstance(v, dict):
            cfg_dict[k] = recipe_to_dict(v, depth-1)
        elif isinstance(v, (int, float, str, bool)):
            cfg_dict[k] = v
        else:
            cfg_dict[k] = str(v)
    return cfg_dict
                    
recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=32,
        micro_batch_size=4,
        num_workers=0,
    )

    cfg_dict = recipe_to_dict(recipe)
    recipe.log.wandb = run.Config(
        WandbLogger,
        name=NAME,
        project="crowd-pilot-nemo",
        save_dir="/fast/project/HFMI_SynergyUnit/mihir/logs/wandb",
        entity="instant-uv",
        config=cfg_dict,
    )
    
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/recipe.log (log)
  model=<Config[Qwen3Model(config=<Config[Qwen3Config600M()]>)]>,
  data=<Config[ChatDataModule(
    dataset_root='/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/',
    seq_length=32768,
    micro_batch_size=1,
    global_batch_size=4)]>,
  trainer=<Config[Trainer(
    accelerator='gpu',
    accumulate_grad_batches=1,
    devices=1,
    limit_test_batches=None,
    limit_val_batches=None,
    log_every_n_steps=1,
    max_steps=1000,
    num_nodes=1,
    plugins=<Config[MegatronMixedPrecision(
      precision='bf16-mixed',
      params_dtype=torch.bfloat16,
      pipeline_dtype=torch.bfloat16,
      autocast_enabled=False,
      grad_reduce_in_fp32=True)]>,
    strategy=<Config[MegatronStrategy(
      tensor_model_parallel_size=1,
      pipeline_model_parallel_size=1,
      virtual_pipeline_model_parallel_size=None,
      context_parallel_size=1,
      sequence_parallel=False,
      pipeline_dtype=torch.bfloat16,
      ckpt_load_strictness='log_all',
      gradient_as_bucket_view=True)]>,
    use_distributed_sampler=False,
    val_check_interval=30,
    callbacks=[<Config[TimingCallback()]>])]>,
  log=<Config[NeMoLogger(
    name='qwen3_600m_full_finetune_crowd_code_dataset_32k_tokens_batch_size_1',
    log_dir='/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs',
    ckpt=<Config[ModelCheckpoint(
      save_last='link',
      save_top_k=2,
      every_n_train_steps=50,
      filename='{model_name}--{val_loss:.2f}-{step}-{consumed_samples}')]>,
    tensorboard=<Config[TensorBoardLogger(
      save_dir='tb_logs',
      name='qwen3_600m_full_finetune_crowd_code_dataset_32k_tokens_batch_size_1')]>,
    wandb=None)]>,
  resume=<Config[AutoResume(
    restore_config=<Config[RestoreConfig(
      path='/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/')]>)]>,
  optim=<Config[MegatronOptimizerModule(
    config=<Config[OptimizerConfig(
      optimizer='adam',
      lr=5e-06,
      weight_decay=0.1,
      fp16=False,
      bf16=True,
      adam_beta1=0.9,
      adam_beta2=0.98,
      adam_eps=1e-05,
      use_distributed_optimizer=True,
      clip_grad=1.0)]>,
    lr_scheduler=<Config[CosineAnnealingScheduler(warmup_steps=50, constant_steps=0, min_lr=0)]>)]>,
  tokenizer='model')]>
[content] slurm/dev/mihir/berlin/crowd_pilot/nemo/iterating_on_stuff/test_full_finetune_4k.py (python)

    MAX_STEPS = 160_000
    NUM_CHECKPOINTS = 10
    recipe.trainer.max_steps = MAX_STEPS
    recipe.log.ckpt.every_n_train_steps = MAX_STEPS // NUM_CHECKPOINTS
    recipe.log.ckpt.save_top_k = NUM_CHECKPOINTS
[tab] slurm/dev/mihir/berlin/crowd_pilot/nemo/qwen_600m/test_full_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm
from lightning.pytorch.loggers import WandbLogger

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"
slurm_job_id = os.environ.get("SLURM_JOB_ID", "local")

NAME = f"qwen3_600m_full_4k_tokens_{slurm_job_id}"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=32,
        micro_batch_size=4,
        num_workers=0,
    )
    
    recipe.log.wandb = run.Config(
        WandbLogger,
        name=NAME,
        project="crowd-pilot-nemo",
        save_dir="/fast/project/HFMI_SynergyUnit/mihir/logs/wandb",
        entity="instant-uv",
    )
    
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()