p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_30b_a3b_lora_finetune_crowd_code_dataset_4k_tokens"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/mihir/logs",
    num_nodes=1,
    num_gpus_per_node=2,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
        num_workers=0,
    )

    # Enable activation checkpointing to fit 30B model on two H100s
    recipe.trainer.strategy.tensor_model_parallel_size = 2
    recipe.trainer.strategy.expert_model_parallel_size = 2
    recipe.model.config.recompute_granularity = "full"
    recipe.model.config.recompute_method = "uniform"
    recipe.model.config.recompute_num_layers = 1
    recipe.trainer.strategy.ckpt_async_save = False 

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/alfred/berlin/tab_model/nemo/setup_nemo_uv.sh (shellscript)
#!/bin/bash

set -ex

export CUDA_HOME=/usr/local/cuda-12.8
export PATH=$CUDA_HOME/bin:$PATH
export LD_LIBRARY_PATH=$CUDA_HOME/lib64:$LD_LIBRARY_PATH


echo "--------------------------------"
echo "Setting up nemo environment..."
echo "--------------------------------"
uv venv .uv_venv --python 3.10.12 --seed
source .uv_venv/bin/activate

echo "--------------------------------"
echo "Installing torch..."
echo "--------------------------------"
uv pip install torch==2.8.0 torchvision torchaudio 

echo "--------------------------------"
echo "Installing nemo toolkit..."
echo "--------------------------------"
uv pip install nemo_toolkit['all']==2.5.0

echo "--------------------------------"
echo "Installing transformer engine..."
echo "--------------------------------"
uv pip install --no-build-isolation transformer_engine[pytorch]==2.7.0
[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/finetune_lora_4k.log (log)
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/crowd-pilot/slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py", line 3, in <module>
    import nemo_run as run
ModuleNotFoundError: No module named 'nemo_run'
srun: error: hai003: task 0: Exited with exit code 1
[tab] /fast/project/HFMI_SynergyUnit/mihir/logs/finetune_lora_4k_34569.log (log)
/fast/home/mihir.mahajan/Projects/crowd-pilot/.uv_venv/lib/python3.10/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
  import pynvml  # type: ignore[import]
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/test_lora_finetune_4k.py (python)
    # Increase learning rate for LoRA (typically 10x higher than full fine-tuning)
    recipe.optimizer.lr = 1e-4  # or try 5e-4, 2e-4
    
    # Increase LoRA rank for better capacity
    recipe.model.peft.lora_rank = 64  # or try 32, 128
    recipe.model.peft.lora_alpha = 128  # typically 2x the rank