p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
import nemo_run as run

from nemo.collections import llm


def configure_recipe(nodes: int = 1, gpus_per_node: int = 2):
    recipe = llm.nemotron3_4b.pretrain_recipe(
        dir="/checkpoints/nemotron", # Path to store checkpoints
        name="nemotron_pretraining",
        tensor_parallelism=2,
        num_nodes=nodes,
        num_gpus_per_node=gpus_per_node,
        max_steps=100, # Setting a small value for the quickstart
    )

    recipe.trainer.val_check_interval = 100
    return recipe

def local_executor_torchrun(nodes: int = 1, devices: int = 2) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_pretraining():
    recipe = configure_recipe()
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name="nemotron3_4b_pretraining")

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_pretraining()
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
@run.cli.factory(target=finetune, name=NAME)
def finetune_recipe(
    dir: Optional[str] = None,
    name: str = "default",
    num_nodes: int = 1,
    num_gpus_per_node: int = 8,
    peft_scheme: Optional[str] = 'lora',
    packed_sequence: bool = False,
) -> run.Partial:
    """
    Create a fine-tuning recipe for Qwen3 30B-A3B model.

    This function sets up a complete configuration for fine-tuning, including
    model, trainer, data, logging, optimization, and resumption settings.
    The recipe uses LoRA (Low-Rank Adaptation) for efficient fine-tuning, unless peft_scheme is set to None.
    This model uses Mixture of Experts (MoE) architecture with 128 experts.

    Args:
        dir (Optional[str]): Directory for saving logs and checkpoints.
        name (str): Name of the fine-tuning run.
        num_nodes (int): Number of compute nodes to use.
        num_gpus_per_node (int): Number of GPUs per node.
        peft_scheme (Optional[str]): Name of the peft scheme to use for fine-tuning.
            Allowed values: 'lora'/'dora'/'none'/None.
        packed_sequence (Optional[bool]): Packing multiple training sequences into one long sequence for training
            efficiency. Default sequence length is 2048.

    Returns:
        run.Partial: Partial configuration for fine-tuning.

    Examples:
        CLI usage:
            $ nemo llm finetune --factory qwen3_30b_a3b

        Python API usage:
            >>> recipe = finetune_recipe(name="qwen3_30b_a3b_finetune", num_nodes=2)
            >>> print(recipe)

    Note:
        This recipe uses the SQuAD dataset for fine-tuning.
    """
    recipe = default_finetune_recipe(
        model(), "Qwen/Qwen3-30B-A3B", dir, name, num_nodes, num_gpus_per_node, packed_sequence
    )
    if peft_scheme is None or peft_scheme.lower() == 'none':
        recipe.trainer.strategy.tensor_model_parallel_size = 4
        recipe.trainer.strategy.expert_model_parallel_size = 4
        recipe.trainer.strategy.expert_tensor_parallel_size = 1
        recipe.trainer.strategy.pipeline_model_parallel_size = 2
        recipe.trainer.strategy.sequence_parallel = True
        recipe.optim.config.lr = 5e-6
    elif peft_scheme.lower() in ['lora', 'dora']:
        recipe.trainer.strategy.tensor_model_parallel_size = 4
        recipe.trainer.strategy.expert_model_parallel_size = 4
        recipe.trainer.strategy.expert_tensor_parallel_size = 1
        recipe.trainer.strategy.sequence_parallel = True
        recipe.peft = run.Config(PEFT_STR2CLS[peft_scheme.lower()])
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
        recipe.peft.target_modules = ['linear_qkv', 'linear_proj']
        recipe.optim.config.lr = 1e-4
    else:
        raise ValueError(f"Unrecognized peft scheme: {peft_scheme}")
    return recipe
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
from typing import Optional

import lightning.pytorch as pl
import nemo_run as run
import torch
from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer

from nemo.collections.llm.api import finetune, pretrain
from nemo.collections.llm.gpt.data.mock import MockDataModule
from nemo.collections.llm.peft import PEFT_STR2CLS
from nemo.collections.llm.recipes.finetune_default import default_finetune_recipe
from nemo.collections.llm.recipes.log.default import default_log, default_resume, tensorboard_logger
from nemo.collections.llm.recipes.optim.adam import distributed_fused_adam_with_cosine_annealing
from nemo.collections.llm.recipes.qwen3 import qwen3_model, qwen3_trainer
from nemo.utils.exp_manager import TimingCallback
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
@run.cli.factory(name=NAME)
def model() -> run.Config[pl.LightningModule]:
    """
    Factory function to create a Qwen3 30B-A3B model configuration.
    This is a MoE (Mixture of Experts) model with 128 experts.

    Returns:
        run.Config[pl.LightningModule]: Configuration for the Qwen3 30B-A3B model.

    Examples:
        CLI usage:
            $ nemo llm pretrain model=qwen3_30b_a3b ...

        Python API usage:
            >>> model_config = model()
            >>> print(model_config)
    """
    return qwen3_model(version=NAME)