p-doom/crowd-code-dataset-0.2 huggingface dataset:
[terminal_command] TERMINAL ()
source /home/franz.srambical/crowd-pilot/.venv/bin/activate[franz.srambical@hai-login2.haicore.berlin:~/crowd-pilot] $ source /home/franz.srambical/crowd-pilot/.venv/bin/activate(crowd-pilot) [franz.srambical@hai-login2.haicore.berlin:~/crowd-pilot] $ less /fast/project/HFMI_SynergyUnit/tab_model/logs/franz/qwen3_600m_finetune_34qwen3_600m_finetune_34530.log  qwen3_600m_finetune_34574.log  qwen3_600m_finetune_34582.log  qwen3_600m_finetune_34661.logqwen3_600m_finetune_34531.log  qwen3_600m_finetune_34575.log  qwen3_600m_finetune_34583.log  qwen3_600m_finetune_34662.logqwen3_600m_finetune_34532.log  qwen3_600m_finetune_34576.log  qwen3_600m_finetune_34586.log  qwen3_600m_finetune_34663.logqwen3_600m_finetune_34566.log  qwen3_600m_finetune_34577.log  qwen3_600m_finetune_34624.log  qwen3_600m_finetune_34664.logqwen3_600m_finetune_34567.log  qwen3_600m_finetune_34578.log  qwen3_600m_finetune_34625.log  qwen3_600m_finetune_34665.logqwen3_600m_finetune_34570.log  qwen3_600m_finetune_34579.log  qwen3_600m_finetune_34629.log  qwen3_600m_finetune_34666.logqwen3_600m_finetune_34571.log  qwen3_600m_finetune_34580.log  qwen3_600m_finetune_34659.log  qwen3_600m_finetune_34672.logqwen3_600m_finetune_34573.log  qwen3_600m_finetune_34581.log  qwen3_600m_finetune_34660.log  (crowd-pilot) [franz.srambical@hai-login2.haicore.berlin:~/crowd-pilot] $ less /fast/project/HFMI_SynergyUnit/tab_model/logs/franz/qwen3_600m_finetune_34660.log
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 16000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 16000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_4k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 16000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_4k copy 4.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 16000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_4k copy 2.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 16000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_4k copy 2.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 16000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_8k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 16000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_8k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_4k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_4k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=4096,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 16000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_8k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_8k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=8192,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 8000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_8k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_8k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=8192,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 8000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_16k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_8k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=8192,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 8000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_16k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_8k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_8k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=8192,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 8000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_16k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_16k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 4000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_16k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_16k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 4000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_32k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_16k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 4000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_32k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_16k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_16k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=16384,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 4000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_32k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_32k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 2000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_32k copy.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_32k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 2000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/lora_bs_1_seqlen_128k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"

NAME = "qwen3_600m_lora_finetune_crowd_code_dataset_32k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 

recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=1,
        micro_batch_size=1,
    )

    recipe.trainer.strategy.tensor_model_parallel_size = 1
    recipe.trainer.strategy.expert_model_parallel_size = 1
    recipe.trainer.max_steps = 2000

    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/qwen3_600m/batch_size_1_varying_seqlens_run/full_bs_1_seqlen_128k.py (python)
import os

import nemo_run as run
from nemo.collections import llm

os.environ["NEMORUN_HOME"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemorun_home"
os.environ["NEMO_CACHE_DIR"] = "/fast/project/HFMI_SynergyUnit/tab_model/franz/nemo_cache"


NAME = "qwen3_600m_full_finetune_crowd_code_dataset_32k_tokens_batch_size_1"
DATASET_DIR = "/fast/project/HFMI_SynergyUnit/tab_model/data/nemo_hf_part_jsonl_32k_tokens/" 


recipe = llm.qwen3_600m.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/tab_model/data/outputs",
    num_nodes=1,
    num_gpus_per_node=1,
    peft_scheme=None,
    # FIXME(f.srambical): check whether this is supported by ChatDataModule
    packed_sequence=False,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-600m/"
    recipe.data = run.Config(
        llm.ChatDataModule,
        dataset_root=DATASET_DIR,
        seq_length=32768,
        global_batch_size=1,
        micro_batch_size=1,
    )
    recipe.trainer.max_steps = 2000
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()